{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "补充学习第二bs4的内容"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<tag>.name tag为标签 name为标签的名字\n",
    "<tag>.attrs; 标签的属性 字典形式组织\n",
    "<tag>.string; 标签内非属性字符串"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html>\n",
      " <head>\n",
      "  <title>\n",
      "   This is a python demo page\n",
      "  </title>\n",
      " </head>\n",
      " <body>\n",
      "  <p class=\"title\">\n",
      "   <b>\n",
      "    The demo python introduces several python courses.\n",
      "   </b>\n",
      "  </p>\n",
      "  <p class=\"course\">\n",
      "   Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\n",
      "   <a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">\n",
      "    Basic Python\n",
      "   </a>\n",
      "   and\n",
      "   <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">\n",
      "    Advanced Python\n",
      "   </a>\n",
      "   .\n",
      "  </p>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "#基本操作\n",
    "from bs4 import BeautifulSoup\n",
    "import requests # 抓取页面\n",
    "\n",
    "r = requests.get('https://python123.io/ws/demo.html') # Demo网址\n",
    "demo = r.text  # 抓取的数据\n",
    "\n",
    "# 解析HTML页面\n",
    "soup = BeautifulSoup(demo, 'html.parser')  # 抓取的页面数据；bs4的解析器\n",
    "# 有层次感的输出解析后的HTML页面\n",
    "print(soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.a #访问soup之后标签的全部内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.a.name#访问标签的名字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'href': 'http://www.icourse163.org/course/BIT-268001',\n",
       " 'class': ['py1'],\n",
       " 'id': 'link1'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.a.attrs#访问标签的全部属性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'link1'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.a.attrs['id']#访问标签的某一个属性值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The demo python introduces several python courses.'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.b.string #访问标签内的非属性字符串 可以理解为访问标签内的文本内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html>\n",
      " <head>\n",
      "  <title>\n",
      "   This is a python demo page\n",
      "  </title>\n",
      " </head>\n",
      " <body>\n",
      "  <p class=\"title\">\n",
      "   <b>\n",
      "    The demo python introduces several python courses.\n",
      "   </b>\n",
      "  </p>\n",
      "  <p class=\"course\">\n",
      "   Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\n",
      "   <a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">\n",
      "    Basic Python\n",
      "   </a>\n",
      "   and\n",
      "   <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">\n",
      "    Advanced Python\n",
      "   </a>\n",
      "   .\n",
      "  </p>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "print(soup.prettify())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "标签树的下行遍历"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<html><head><title>This is a python demo page</title></head>\n",
       " <body>\n",
       " <p class=\"title\"><b>The demo python introduces several python courses.</b></p>\n",
       " <p class=\"course\">Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\n",
       " <a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a> and <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">Advanced Python</a>.</p>\n",
       " </body></html>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.contents#获取整个标签树的儿子节点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\n',\n",
       " <p class=\"title\"><b>The demo python introduces several python courses.</b></p>,\n",
       " '\\n',\n",
       " <p class=\"course\">Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\n",
       " <a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a> and <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">Advanced Python</a>.</p>,\n",
       " '\\n']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.body.contents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "<p class=\"title\"><b>The demo python introduces several python courses.</b></p>\n",
      "\n",
      "\n",
      "<p class=\"course\">Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\r\n",
      "<a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a> and <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">Advanced Python</a>.</p>\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for child in soup.body.children:#遍历儿子节点\n",
    "    print(child)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "标签树的平行遍历 必须满足条件一：必须是同一个父节点 条件二：标签中的内容也构成了节点，同一个父节点之中的内容"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "查找方法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "find_all(name, attrs, recursive, string, **kwargs)\n",
    "第三个 默认对子孙全部进行检索\n",
    "（）等价于find_all\n",
    "string是对<>…</>中字符串区域的检索字符串\n",
    "string可单独作为一个函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"title\"><b>The demo python introduces several python courses.</b></p>,\n",
       " <p class=\"course\">Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\n",
       " <a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a> and <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">Advanced Python</a>.</p>,\n",
       " <a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a>,\n",
       " <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">Advanced Python</a>]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all(['a','p'])#对两个节点进行检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all(id=\"link\") # 完全匹配才能匹配到"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Basic Python']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all(string = \"Basic Python\") # 完全匹配才能匹配到"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "实战 顶尖人才排名爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['1', '清华大学', '94.6'],\n",
       " ['2', '北京大学', '76.5'],\n",
       " ['3', '浙江大学', '72.9'],\n",
       " ['4', '上海交通大学', '72.1'],\n",
       " ['5', '复旦大学', '65.6'],\n",
       " ['6', '中国科学技术大学', '60.9'],\n",
       " ['7', '华中科技大学', '58.9'],\n",
       " ['7', '南京大学', '58.9'],\n",
       " ['9', '中山大学', '58.2'],\n",
       " ['10', '哈尔滨工业大学', '56.7'],\n",
       " ['11', '北京航空航天大学', '56.3'],\n",
       " ['12', '武汉大学', '56.2'],\n",
       " ['13', '同济大学', '55.7'],\n",
       " ['14', '西安交通大学', '55.0'],\n",
       " ['15', '四川大学', '54.4'],\n",
       " ['16', '北京理工大学', '54.0'],\n",
       " ['17', '东南大学', '53.6'],\n",
       " ['18', '南开大学', '52.8'],\n",
       " ['19', '天津大学', '52.3'],\n",
       " ['20', '华南理工大学', '52.0'],\n",
       " ['21', '中南大学', '50.3'],\n",
       " ['22', '北京师范大学', '49.7'],\n",
       " ['23', '山东大学', '49.1'],\n",
       " ['23', '厦门大学', '49.1'],\n",
       " ['25', '吉林大学', '48.9'],\n",
       " ['26', '大连理工大学', '48.6'],\n",
       " ['27', '电子科技大学', '48.4'],\n",
       " ['28', '湖南大学', '48.1'],\n",
       " ['29', '苏州大学', '47.3'],\n",
       " ['30', '西北工业大学', '46.7'],\n",
       " ['31', '中国人民大学', '46.1'],\n",
       " ['32', '华东师范大学', '46.0'],\n",
       " ['33', '南京航空航天大学', '44.8'],\n",
       " ['34', '对外经济贸易大学', '44.7'],\n",
       " ['35', '南方科技大学', '44.6'],\n",
       " ['36', '华东理工大学', '44.5'],\n",
       " ['37', '重庆大学', '44.4'],\n",
       " ['38', '南京理工大学', '44.3'],\n",
       " ['39', '北京科技大学', '43.9'],\n",
       " ['40', '东北大学', '43.7'],\n",
       " ['41', '上海大学', '43.4'],\n",
       " ['42', '北京邮电大学', '42.9'],\n",
       " ['42', '上海财经大学', '42.9'],\n",
       " ['42', '中国农业大学', '42.9'],\n",
       " ['45', '武汉理工大学', '42.8'],\n",
       " ['46', '北京交通大学', '42.6'],\n",
       " ['46', '华中师范大学', '42.6'],\n",
       " ['48', '西安电子科技大学', '42.1'],\n",
       " ['49', '中央财经大学', '41.8'],\n",
       " ['50', '北京化工大学', '41.7'],\n",
       " ['51', '暨南大学', '41.5'],\n",
       " ['52', '东华大学', '41.4'],\n",
       " ['52', '兰州大学', '41.4'],\n",
       " ['52', '中南财经政法大学', '41.4'],\n",
       " ['55', '宁波诺丁汉大学', '40.3'],\n",
       " ['56', '哈尔滨工程大学', '40.1'],\n",
       " ['56', '华北电力大学', '40.1'],\n",
       " ['58', '深圳大学', '40.0'],\n",
       " ['59', '江南大学', '39.6'],\n",
       " ['59', '南京师范大学', '39.6'],\n",
       " ['61', '北京工业大学', '39.5'],\n",
       " ['62', '福州大学', '39.4'],\n",
       " ['63', '北京外国语大学', '39.3'],\n",
       " ['63', '北京语言大学', '39.3'],\n",
       " ['63', '中国地质大学（武汉）', '39.3'],\n",
       " ['66', '西南交通大学', '39.2'],\n",
       " ['67', '华中农业大学', '39.1'],\n",
       " ['67', '中国海洋大学', '39.1'],\n",
       " ['69', '南京农业大学', '39.0'],\n",
       " ['70', '上海外国语大学', '38.8'],\n",
       " ['71', '中国矿业大学', '38.6'],\n",
       " ['72', '中国地质大学（北京）', '38.5'],\n",
       " ['73', '东北财经大学', '38.2'],\n",
       " ['73', '西南财经大学', '38.2'],\n",
       " ['73', '西南大学', '38.2'],\n",
       " ['76', '东北师范大学', '38.1'],\n",
       " ['76', '南京邮电大学', '38.1'],\n",
       " ['76', '中国政法大学', '38.1'],\n",
       " ['79', '河海大学', '38.0'],\n",
       " ['80', '南京信息工程大学', '37.9'],\n",
       " ['81', '西北农林科技大学', '37.8'],\n",
       " ['82', '中国石油大学（华东）', '37.4'],\n",
       " ['83', '合肥工业大学', '37.3'],\n",
       " ['84', '陕西师范大学', '37.2'],\n",
       " ['85', '华南师范大学', '37.1'],\n",
       " ['85', '江苏大学', '37.1'],\n",
       " ['87', '南京工业大学', '37.0'],\n",
       " ['87', '中国石油大学（北京）', '37.0'],\n",
       " ['89', '西北大学', '36.9'],\n",
       " ['89', '浙江工业大学', '36.9'],\n",
       " ['91', '北京林业大学', '36.8'],\n",
       " ['91', '湖南师范大学', '36.8'],\n",
       " ['91', '浙江师范大学', '36.8'],\n",
       " ['94', '首都师范大学', '36.4'],\n",
       " ['95', '汕头大学', '36.3'],\n",
       " ['96', '中国传媒大学', '36.2'],\n",
       " ['97', '杭州电子科技大学', '36.1'],\n",
       " ['98', '扬州大学', '36.0'],\n",
       " ['99', '安徽大学', '35.9'],\n",
       " ['100', '华侨大学', '35.7'],\n",
       " ['101', '宁波大学', '35.4'],\n",
       " ['101', '首都经济贸易大学', '35.4'],\n",
       " ['101', '西交利物浦大学', '35.4'],\n",
       " ['104', '燕山大学', '35.3'],\n",
       " ['105', '湖北大学', '35.2'],\n",
       " ['106', '长安大学', '35.1'],\n",
       " ['106', '上海理工大学', '35.1'],\n",
       " ['108', '大连海事大学', '35.0'],\n",
       " ['108', '广东外语外贸大学', '35.0'],\n",
       " ['108', '太原理工大学', '35.0'],\n",
       " ['111', '广东工业大学', '34.8'],\n",
       " ['111', '南京财经大学', '34.8'],\n",
       " ['111', '浙江理工大学', '34.8'],\n",
       " ['114', '河北工业大学', '34.7'],\n",
       " ['115', '华东政法大学', '34.6'],\n",
       " ['115', '湘潭大学', '34.6'],\n",
       " ['115', '浙江财经大学', '34.6'],\n",
       " ['115', '中国矿业大学（北京）', '34.6'],\n",
       " ['119', '青岛大学', '34.5'],\n",
       " ['120', '上海师范大学', '34.2'],\n",
       " ['121', '南京审计大学', '34.1'],\n",
       " ['122', '华南农业大学', '34.0'],\n",
       " ['123', '江西财经大学', '33.9'],\n",
       " ['123', '辽宁大学', '33.9'],\n",
       " ['123', '中央民族大学', '33.9'],\n",
       " ['126', '武汉工程大学', '33.8'],\n",
       " ['126', '郑州大学', '33.8'],\n",
       " ['128', '杭州师范大学', '33.6'],\n",
       " ['128', '上海对外经贸大学', '33.6'],\n",
       " ['130', '上海海事大学', '33.4'],\n",
       " ['131', '山东师范大学', '33.3'],\n",
       " ['131', '武汉科技大学', '33.3'],\n",
       " ['133', '北京第二外国语学院', '33.2'],\n",
       " ['133', '广西大学', '33.2'],\n",
       " ['133', '云南大学', '33.2'],\n",
       " ['133', '浙江工商大学', '33.2'],\n",
       " ['137', '山西大学', '33.1'],\n",
       " ['138', '东北农业大学', '33.0'],\n",
       " ['139', '广州大学', '32.7'],\n",
       " ['139', '黑龙江大学', '32.7'],\n",
       " ['139', '南昌大学', '32.7'],\n",
       " ['139', '天津财经大学', '32.7'],\n",
       " ['143', '东北林业大学', '32.5'],\n",
       " ['143', '河南大学', '32.5'],\n",
       " ['145', '河北大学', '32.4'],\n",
       " ['146', '江苏师范大学', '32.3'],\n",
       " ['147', '济南大学', '32.0'],\n",
       " ['147', '山东财经大学', '32.0'],\n",
       " ['149', '山东科技大学', '31.9'],\n",
       " ['150', '长沙理工大学', '31.7'],\n",
       " ['151', '西南政法大学', '31.4'],\n",
       " ['152', '渤海大学', '31.3'],\n",
       " ['152', '重庆工商大学', '31.3'],\n",
       " ['152', '福建师范大学', '31.3'],\n",
       " ['152', '天津师范大学', '31.3'],\n",
       " ['156', '曲阜师范大学', '31.2'],\n",
       " ['156', '三峡大学', '31.2'],\n",
       " ['158', '北京工商大学', '31.1'],\n",
       " ['158', '西安理工大学', '31.1'],\n",
       " ['158', '浙江农林大学', '31.1'],\n",
       " ['161', '广西师范大学', '31.0'],\n",
       " ['161', '温州大学', '31.0'],\n",
       " ['163', '湖北工业大学', '30.9'],\n",
       " ['163', '四川农业大学', '30.9'],\n",
       " ['165', '重庆邮电大学', '30.8'],\n",
       " ['165', '河北师范大学', '30.8'],\n",
       " ['165', '河南师范大学', '30.8'],\n",
       " ['168', '北京体育大学', '30.7'],\n",
       " ['168', '常州大学', '30.7'],\n",
       " ['168', '天津工业大学', '30.7'],\n",
       " ['171', '石家庄铁道大学', '30.6'],\n",
       " ['172', '中国计量大学', '30.4'],\n",
       " ['173', '海南大学', '30.3'],\n",
       " ['173', '江苏科技大学', '30.3'],\n",
       " ['175', '安徽财经大学', '30.2'],\n",
       " ['175', '北京建筑大学', '30.2'],\n",
       " ['175', '中南民族大学', '30.2'],\n",
       " ['178', '安徽工业大学', '30.1'],\n",
       " ['178', '北京物资学院', '30.1'],\n",
       " ['178', '南通大学', '30.1'],\n",
       " ['178', '内蒙古大学', '30.1'],\n",
       " ['178', '中国人民公安大学', '30.1'],\n",
       " ['183', '青岛科技大学', '30.0'],\n",
       " ['183', '西南石油大学', '30.0'],\n",
       " ['185', '武汉纺织大学', '29.9'],\n",
       " ['185', '中国民航大学', '29.9'],\n",
       " ['187', '华北理工大学', '29.8'],\n",
       " ['187', '南京林业大学', '29.8'],\n",
       " ['187', '西安建筑科技大学', '29.8'],\n",
       " ['190', '北方工业大学', '29.7'],\n",
       " ['190', '河北科技大学', '29.7'],\n",
       " ['190', '上海电力大学', '29.7'],\n",
       " ['193', '安徽师范大学', '29.6'],\n",
       " ['193', '哈尔滨师范大学', '29.6'],\n",
       " ['193', '湖南农业大学', '29.6'],\n",
       " ['193', '上海海洋大学', '29.6'],\n",
       " ['193', '上海立信会计金融学院', '29.6'],\n",
       " ['193', '四川师范大学', '29.6'],\n",
       " ['199', '辽宁工业大学', '29.5'],\n",
       " ['200', '沈阳航空航天大学', '29.4'],\n",
       " ['201', '湖南工业大学', '29.3'],\n",
       " ['202', '江西师范大学', '29.2'],\n",
       " ['202', '西华师范大学', '29.2'],\n",
       " ['204', '福建农林大学', '29.1'],\n",
       " ['204', '天津理工大学', '29.1'],\n",
       " ['206', '安徽农业大学', '29.0'],\n",
       " ['206', '兰州交通大学', '29.0'],\n",
       " ['206', '陕西科技大学', '29.0'],\n",
       " ['209', '集美大学', '28.9'],\n",
       " ['210', '长春理工大学', '28.8'],\n",
       " ['210', '重庆交通大学', '28.8'],\n",
       " ['210', '中南林业科技大学', '28.8'],\n",
       " ['213', '重庆师范大学', '28.7'],\n",
       " ['213', '湖南科技大学', '28.7'],\n",
       " ['213', '江汉大学', '28.7'],\n",
       " ['216', '成都理工大学', '28.6'],\n",
       " ['216', '河南理工大学', '28.6'],\n",
       " ['216', '辽宁师范大学', '28.6'],\n",
       " ['219', '南华大学', '28.5'],\n",
       " ['220', '长江大学', '28.4'],\n",
       " ['220', '重庆理工大学', '28.4'],\n",
       " ['220', '武汉轻工大学', '28.4'],\n",
       " ['220', '云南师范大学', '28.4'],\n",
       " ['224', '闽南师范大学', '28.3'],\n",
       " ['224', '山西财经大学', '28.3'],\n",
       " ['226', '河南工业大学', '28.2'],\n",
       " ['226', '湖州师范学院', '28.2'],\n",
       " ['226', '西安邮电大学', '28.2'],\n",
       " ['229', '广西民族大学', '28.1'],\n",
       " ['229', '湖北经济学院', '28.1'],\n",
       " ['229', '山西师范大学', '28.1'],\n",
       " ['229', '苏州科技大学', '28.1'],\n",
       " ['229', '天津科技大学', '28.1'],\n",
       " ['234', '湖北第二师范学院', '28.0'],\n",
       " ['234', '吉林师范大学', '28.0'],\n",
       " ['234', '郑州师范学院', '28.0'],\n",
       " ['237', '哈尔滨理工大学', '27.9'],\n",
       " ['237', '河北农业大学', '27.9'],\n",
       " ['237', '吉首大学', '27.9'],\n",
       " ['240', '北京信息科技大学', '27.8'],\n",
       " ['240', '西北师范大学', '27.8'],\n",
       " ['240', '浙江外国语学院', '27.8'],\n",
       " ['240', '中北大学', '27.8'],\n",
       " ['244', '贵州大学', '27.7'],\n",
       " ['244', '湖南商学院', '27.7'],\n",
       " ['246', '河南财经政法大学', '27.6'],\n",
       " ['246', '淮北师范大学', '27.6'],\n",
       " ['246', '沈阳农业大学', '27.6'],\n",
       " ['246', '浙江海洋大学', '27.6'],\n",
       " ['250', '东莞理工学院', '27.5'],\n",
       " ['250', '湖北师范大学', '27.5'],\n",
       " ['250', '昆明理工大学', '27.5'],\n",
       " ['250', '西南民族大学', '27.5'],\n",
       " ['254', '上海第二工业大学', '27.4'],\n",
       " ['254', '上海应用技术大学', '27.4'],\n",
       " ['256', '绍兴文理学院', '27.3'],\n",
       " ['257', '东北电力大学', '27.2'],\n",
       " ['257', '桂林电子科技大学', '27.2'],\n",
       " ['257', '浙江传媒学院', '27.2'],\n",
       " ['260', '河北经贸大学', '27.1'],\n",
       " ['260', '河南农业大学', '27.1'],\n",
       " ['262', '东北石油大学', '27.0'],\n",
       " ['262', '石河子大学', '27.0'],\n",
       " ['264', '兰州理工大学', '26.9'],\n",
       " ['265', '广西财经学院', '26.8'],\n",
       " ['265', '西安工业大学', '26.8'],\n",
       " ['265', '浙江科技学院', '26.8'],\n",
       " ['268', '安徽理工大学', '26.7'],\n",
       " ['268', '大连民族大学', '26.7'],\n",
       " ['268', '新疆大学', '26.7'],\n",
       " ['268', '烟台大学', '26.7'],\n",
       " ['268', '延边大学', '26.7'],\n",
       " ['273', '长沙学院', '26.6'],\n",
       " ['273', '大连交通大学', '26.6'],\n",
       " ['275', '吉林财经大学', '26.5'],\n",
       " ['275', '青岛理工大学', '26.5'],\n",
       " ['275', '上海工程技术大学', '26.5'],\n",
       " ['275', '西安科技大学', '26.5'],\n",
       " ['279', '长春工业大学', '26.4'],\n",
       " ['279', '桂林理工大学', '26.4'],\n",
       " ['279', '河南科技大学', '26.4'],\n",
       " ['279', '淮阴师范学院', '26.4'],\n",
       " ['279', '西南科技大学', '26.4'],\n",
       " ['284', '佛山科学技术学院', '26.3'],\n",
       " ['284', '南京工程学院', '26.3'],\n",
       " ['284', '宁夏大学', '26.3'],\n",
       " ['287', '广东技术师范大学', '26.2'],\n",
       " ['287', '河北工程大学', '26.2'],\n",
       " ['287', '黄冈师范学院', '26.2'],\n",
       " ['287', '南京晓庄学院', '26.2'],\n",
       " ['287', '沈阳建筑大学', '26.2'],\n",
       " ['287', '西安工程大学', '26.2'],\n",
       " ['293', '大理大学', '26.1'],\n",
       " ['293', '大连大学', '26.1'],\n",
       " ['295', '湖北民族大学', '25.9'],\n",
       " ['295', '华东交通大学', '25.9'],\n",
       " ['295', '嘉兴学院', '25.9'],\n",
       " ['295', '闽江学院', '25.9'],\n",
       " ['295', '青海大学', '25.9'],\n",
       " ['300', '大连工业大学', '25.8'],\n",
       " ['300', '河北地质大学', '25.8'],\n",
       " ['300', '天津商业大学', '25.8'],\n",
       " ['300', '玉林师范学院', '25.8'],\n",
       " ['300', '中国人民武装警察部队学院', '25.8'],\n",
       " ['305', '北京印刷学院', '25.7'],\n",
       " ['305', '广东金融学院', '25.7'],\n",
       " ['305', '南宁师范大学', '25.7'],\n",
       " ['305', '沈阳师范大学', '25.7'],\n",
       " ['305', '厦门理工学院', '25.7'],\n",
       " ['310', '成都信息工程大学', '25.5'],\n",
       " ['310', '沈阳工业大学', '25.5'],\n",
       " ['310', '云南财经大学', '25.5'],\n",
       " ['313', '安徽建筑大学', '25.4'],\n",
       " ['313', '哈尔滨商业大学', '25.4'],\n",
       " ['313', '黑龙江工程学院', '25.4'],\n",
       " ['313', '湖北工程学院', '25.4'],\n",
       " ['313', '江苏理工学院', '25.4'],\n",
       " ['313', '郑州轻工业大学', '25.4'],\n",
       " ['319', '长春师范大学', '25.3'],\n",
       " ['319', '湖南第一师范学院', '25.3'],\n",
       " ['319', '吉林农业大学', '25.3'],\n",
       " ['319', '太原科技大学', '25.3'],\n",
       " ['319', '太原师范学院', '25.3'],\n",
       " ['324', '常熟理工学院', '25.2'],\n",
       " ['324', '广东石油化工学院', '25.2'],\n",
       " ['324', '海南师范大学', '25.2'],\n",
       " ['324', '华北水利水电大学', '25.2'],\n",
       " ['324', '宁波工程学院', '25.2'],\n",
       " ['324', '西安石油大学', '25.2'],\n",
       " ['324', '西华大学', '25.2'],\n",
       " ['331', '河北科技师范学院', '25.1'],\n",
       " ['331', '湖南城市学院', '25.1'],\n",
       " ['331', '内蒙古工业大学', '25.1'],\n",
       " ['331', '山东农业大学', '25.1'],\n",
       " ['331', '延安大学', '25.1'],\n",
       " ['336', '大连海洋大学', '25.0'],\n",
       " ['336', '广东第二师范学院', '25.0'],\n",
       " ['336', '河北北方学院', '25.0'],\n",
       " ['336', '五邑大学', '25.0'],\n",
       " ['336', '西安财经大学', '25.0'],\n",
       " ['341', '重庆文理学院', '24.9'],\n",
       " ['341', '贵州师范学院', '24.9'],\n",
       " ['343', '宝鸡文理学院', '24.8'],\n",
       " ['343', '南昌航空大学', '24.8'],\n",
       " ['343', '山西大同大学', '24.8'],\n",
       " ['343', '山西农业大学', '24.8'],\n",
       " ['343', '信阳师范学院', '24.8'],\n",
       " ['348', '广东海洋大学', '24.7'],\n",
       " ['349', '湖南工程学院', '24.6'],\n",
       " ['350', '山东理工大学', '24.5'],\n",
       " ['351', '北华航天工业学院', '24.4'],\n",
       " ['351', '湖北汽车工业学院', '24.4'],\n",
       " ['351', '湖南理工学院', '24.4'],\n",
       " ['351', '浙江大学城市学院', '24.4'],\n",
       " ['355', '聊城大学', '24.3'],\n",
       " ['355', '中国民用航空飞行学院', '24.3'],\n",
       " ['357', '安徽工程大学', '24.2'],\n",
       " ['357', '北华大学', '24.2'],\n",
       " ['357', '辽宁工程技术大学', '24.2'],\n",
       " ['357', '南阳师范学院', '24.2'],\n",
       " ['357', '盐城师范学院', '24.2'],\n",
       " ['362', '成都大学', '24.1'],\n",
       " ['362', '福建工程学院', '24.1'],\n",
       " ['362', '陕西理工大学', '24.1'],\n",
       " ['365', '兰州城市学院', '24.0'],\n",
       " ['365', '鲁东大学', '24.0'],\n",
       " ['365', '武汉体育学院', '24.0'],\n",
       " ['365', '徐州工程学院', '24.0'],\n",
       " ['369', '黑龙江科技大学', '23.9'],\n",
       " ['369', '湖北科技学院', '23.9'],\n",
       " ['369', '湖北文理学院', '23.9'],\n",
       " ['372', '北京联合大学', '23.8'],\n",
       " ['372', '江苏第二师范学院', '23.8'],\n",
       " ['372', '江西科技师范大学', '23.8'],\n",
       " ['372', '江西理工大学', '23.8'],\n",
       " ['372', '齐齐哈尔大学', '23.8'],\n",
       " ['372', '韶关学院', '23.8'],\n",
       " ['372', '西安文理学院', '23.8'],\n",
       " ['379', '长春大学', '23.7'],\n",
       " ['379', '洛阳理工学院', '23.7'],\n",
       " ['379', '山东建筑大学', '23.7'],\n",
       " ['382', '沈阳理工大学', '23.6'],\n",
       " ['382', '盐城工学院', '23.6'],\n",
       " ['384', '北京农学院', '23.5'],\n",
       " ['384', '河南科技学院', '23.5'],\n",
       " ['384', '洛阳师范学院', '23.5'],\n",
       " ['384', '内江师范学院', '23.5'],\n",
       " ['384', '沈阳化工大学', '23.5'],\n",
       " ['384', '忻州师范学院', '23.5'],\n",
       " ['384', '运城学院', '23.5'],\n",
       " ['391', '甘肃农业大学', '23.4'],\n",
       " ['391', '广东财经大学', '23.4'],\n",
       " ['391', '湖南财政经济学院', '23.4'],\n",
       " ['391', '湖南文理学院', '23.4'],\n",
       " ['391', '金陵科技学院', '23.4'],\n",
       " ['391', '岭南师范学院', '23.4'],\n",
       " ['391', '内蒙古农业大学', '23.4'],\n",
       " ['391', '齐鲁工业大学', '23.4'],\n",
       " ['391', '上海电机学院', '23.4'],\n",
       " ['391', '上海体育学院', '23.4'],\n",
       " ['401', '辽宁科技大学', '23.3'],\n",
       " ['401', '青岛农业大学', '23.3'],\n",
       " ['401', '新疆师范大学', '23.3'],\n",
       " ['401', '仲恺农业工程学院', '23.3'],\n",
       " ['405', '安康学院', '23.2'],\n",
       " ['405', '赣南师范大学', '23.2'],\n",
       " ['405', '淮海工学院', '23.2'],\n",
       " ['405', '临沂大学', '23.2'],\n",
       " ['405', '泉州师范学院', '23.2'],\n",
       " ['405', '唐山学院', '23.2'],\n",
       " ['405', '通化师范学院', '23.2'],\n",
       " ['412', '北京服装学院', '23.1'],\n",
       " ['412', '合肥学院', '23.1'],\n",
       " ['412', '江西农业大学', '23.1'],\n",
       " ['412', '沈阳大学', '23.1'],\n",
       " ['416', '湖北理工学院', '23.0'],\n",
       " ['416', '淮阴工学院', '23.0'],\n",
       " ['416', '曲靖师范学院', '23.0'],\n",
       " ['419', '安庆师范大学', '22.9'],\n",
       " ['419', '惠州学院', '22.9'],\n",
       " ['419', '佳木斯大学', '22.9'],\n",
       " ['422', '鞍山师范学院', '22.8'],\n",
       " ['422', '衡阳师范学院', '22.8'],\n",
       " ['422', '沈阳工程学院', '22.8'],\n",
       " ['422', '石家庄学院', '22.8'],\n",
       " ['422', '四川轻化工大学', '22.8'],\n",
       " ['422', '肇庆学院', '22.8'],\n",
       " ['428', '黄山学院', '22.7'],\n",
       " ['428', '皖西学院', '22.7'],\n",
       " ['430', '安阳师范学院', '22.6'],\n",
       " ['430', '北京石油化工学院', '22.6'],\n",
       " ['430', '龙岩学院', '22.6'],\n",
       " ['430', '内蒙古科技大学', '22.6'],\n",
       " ['434', '河南工程学院', '22.5'],\n",
       " ['434', '天津城建大学', '22.5'],\n",
       " ['436', '北部湾大学', '22.4'],\n",
       " ['437', '长江师范学院', '22.3'],\n",
       " ['437', '吉林化工学院', '22.3'],\n",
       " ['437', '吉林建筑大学', '22.3'],\n",
       " ['437', '景德镇陶瓷大学', '22.3'],\n",
       " ['437', '云南民族大学', '22.3'],\n",
       " ['442', '辽宁石油化工大学', '22.2'],\n",
       " ['443', '广西科技大学', '22.1'],\n",
       " ['443', '哈尔滨学院', '22.1'],\n",
       " ['443', '山东工商学院', '22.1'],\n",
       " ['443', '太原工业学院', '22.1'],\n",
       " ['443', '郑州航空工业管理学院', '22.1'],\n",
       " ['448', '常州工学院', '22.0'],\n",
       " ['448', '重庆科技学院', '22.0'],\n",
       " ['448', '华北科技学院', '22.0'],\n",
       " ['448', '廊坊师范学院', '22.0'],\n",
       " ['448', '内蒙古民族大学', '22.0'],\n",
       " ['448', '西南林业大学', '22.0'],\n",
       " ['448', '咸阳师范学院', '22.0'],\n",
       " ['448', '湘南学院', '22.0'],\n",
       " ['448', '云南农业大学', '22.0'],\n",
       " ['457', '北方民族大学', '21.9'],\n",
       " ['458', '贵州师范大学', '21.8'],\n",
       " ['458', '天津农学院', '21.8'],\n",
       " ['460', '福建江夏学院', '21.7'],\n",
       " ['460', '合肥师范学院', '21.7'],\n",
       " ['460', '青海师范大学', '21.7'],\n",
       " ['463', '重庆第二师范学院', '21.6'],\n",
       " ['463', '丽水学院', '21.6'],\n",
       " ['463', '南昌工程学院', '21.6'],\n",
       " ['463', '齐鲁师范学院', '21.6'],\n",
       " ['463', '天津职业技术师范大学', '21.6'],\n",
       " ['463', '新乡学院', '21.6'],\n",
       " ['469', '内蒙古师范大学', '21.5'],\n",
       " ['469', '渭南师范学院', '21.5'],\n",
       " ['471', '河南牧业经济学院', '21.4'],\n",
       " ['471', '井冈山大学', '21.4'],\n",
       " ['473', '韩山师范学院', '21.3'],\n",
       " ['473', '河南城建学院', '21.3'],\n",
       " ['473', '贺州学院', '21.3'],\n",
       " ['473', '西北民族大学', '21.3'],\n",
       " ['473', '邢台学院', '21.3'],\n",
       " ['478', '安阳工学院', '21.2'],\n",
       " ['478', '贵州财经大学', '21.2'],\n",
       " ['478', '河西学院', '21.2'],\n",
       " ['478', '昆明学院', '21.2'],\n",
       " ['478', '乐山师范学院', '21.2'],\n",
       " ['483', '山东交通学院', '21.1'],\n",
       " ['483', '玉溪师范学院', '21.1'],\n",
       " ['485', '东华理工大学', '21.0'],\n",
       " ['485', '嘉应学院', '21.0'],\n",
       " ['485', '荆楚理工学院', '21.0'],\n",
       " ['485', '商洛学院', '21.0'],\n",
       " ['489', '贵州民族大学', '20.8'],\n",
       " ['489', '黑龙江八一农垦大学', '20.8'],\n",
       " ['489', '绵阳师范学院', '20.8'],\n",
       " ['489', '牡丹江师范学院', '20.8'],\n",
       " ['489', '浙江万里学院', '20.8'],\n",
       " ['494', '防灾科技学院', '20.6'],\n",
       " ['494', '宜宾学院', '20.6'],\n",
       " ['494', '中原工学院', '20.6'],\n",
       " ['497', '周口师范学院', '20.4'],\n",
       " ['498', '重庆三峡学院', '20.3'],\n",
       " ['498', '济宁学院', '20.3'],\n",
       " ['498', '泰山学院', '20.3'],\n",
       " ['498', '西安航空学院', '20.3'],\n",
       " ['498', '浙江水利水电学院', '20.3'],\n",
       " ['503', '辽东学院', '20.1'],\n",
       " ['503', '新疆农业大学', '20.1'],\n",
       " ['505', '滨州学院', '20.0'],\n",
       " ['506', '长春工程学院', '19.9'],\n",
       " ['507', '赤峰学院', '19.8'],\n",
       " ['507', '南阳理工学院', '19.8'],\n",
       " ['509', '成都工业学院', '19.6'],\n",
       " ['509', '四川文理学院', '19.6'],\n",
       " ['509', '天水师范学院', '19.6'],\n",
       " ['512', '黄淮学院', '19.5'],\n",
       " ['512', '铜陵学院', '19.5'],\n",
       " ['514', '湖南工学院', '19.4'],\n",
       " ['514', '湖南科技学院', '19.4'],\n",
       " ['514', '九江学院', '19.4'],\n",
       " ['514', '邵阳学院', '19.4'],\n",
       " ['514', '许昌学院', '19.4'],\n",
       " ['519', '西藏大学', '19.3'],\n",
       " ['520', '桂林航天工业学院', '19.2'],\n",
       " ['520', '衢州学院', '19.2'],\n",
       " ['520', '商丘师范学院', '19.2'],\n",
       " ['520', '铜仁学院', '19.2'],\n",
       " ['524', '红河学院', '19.0'],\n",
       " ['524', '怀化学院', '19.0'],\n",
       " ['524', '塔里木大学', '19.0'],\n",
       " ['527', '池州学院', '18.9'],\n",
       " ['528', '滁州学院', '18.8'],\n",
       " ['529', '安徽科技学院', '18.7'],\n",
       " ['529', '贵阳学院', '18.7'],\n",
       " ['529', '潍坊学院', '18.7'],\n",
       " ['529', '宜春学院', '18.7'],\n",
       " ['533', '淮南师范学院', '18.6'],\n",
       " ['534', '楚雄师范学院', '18.3'],\n",
       " ['534', '贵州理工学院', '18.3'],\n",
       " ['536', '巢湖学院', '18.2'],\n",
       " ['537', '德州学院', '18.0'],\n",
       " ['538', '攀枝花学院', '17.9'],\n",
       " ['539', '伊犁师范大学', '17.7'],\n",
       " ['539', '榆林学院', '17.7'],\n",
       " ['541', '陇东学院', '17.5'],\n",
       " ['542', '西京学院', '17.3'],\n",
       " ['543', '邯郸学院', '17.2'],\n",
       " ['544', '海南热带海洋学院', '17.1'],\n",
       " ['544', '莆田学院', '17.1'],\n",
       " ['544', '枣庄学院', '17.1'],\n",
       " ['547', '浙江树人学院', '16.7'],\n",
       " ['548', '平顶山学院', '16.0'],\n",
       " ['549', '湖南涉外经济学院', '15.5']]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import bs4\n",
    "def getHTMLText(url):\n",
    "    try:\n",
    "        r = requests.get(url, timeout=30) \n",
    "        r.raise_for_status()\n",
    "        r.encoding = r.apparent_encoding\n",
    "        return r.text\n",
    "    except:\n",
    "        return \"\"\n",
    "\n",
    "def fillUnivList(ulist, html):\n",
    "    soup = BeautifulSoup(html, \"html.parser\")\n",
    "    for tr in soup.find('tbody').children: #find搜索且只返回一个结果\n",
    "        if isinstance(tr, bs4.element.Tag):\n",
    "            tds = tr('td')#等价于find——all\n",
    "            # 根据实际提取需要的内容，\n",
    "            ulist.append([tds[0].string, tds[1].string, tds[3].string]) \n",
    "    return  ulist\n",
    "u_info = [] # 存储爬取结果的容器\n",
    "url = 'http://www.zuihaodaxue.cn/zuihaodaxuepaiming2019.html'\n",
    "\n",
    "html = getHTMLText(url)\n",
    "fillUnivList(u_info, html) # 爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对中英文混排输出问题进行优化:对format()，设定宽度和添加参数chr(12288)\n",
    "def printUnivList(ulist, num=20):\n",
    "    tplt = \"{0:^10}\\t{1:{3}^10}\\t{2:^10}\"\n",
    "    print(tplt.format('排名', '学校名称', '总分', chr(12288)))\n",
    "    for i in range(num):\n",
    "        u = ulist[i]\n",
    "        print(tplt.format(u[0], u[1], u[2], chr(12288)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    排名    \t　　　学校名称　　　\t    总分    \n",
      "    1     \t　　　清华大学　　　\t   94.6   \n",
      "    2     \t　　　北京大学　　　\t   76.5   \n",
      "    3     \t　　　浙江大学　　　\t   72.9   \n",
      "    4     \t　　上海交通大学　　\t   72.1   \n",
      "    5     \t　　　复旦大学　　　\t   65.6   \n",
      "    6     \t　中国科学技术大学　\t   60.9   \n",
      "    7     \t　　华中科技大学　　\t   58.9   \n",
      "    7     \t　　　南京大学　　　\t   58.9   \n",
      "    9     \t　　　中山大学　　　\t   58.2   \n",
      "    10    \t　哈尔滨工业大学　　\t   56.7   \n",
      "    11    \t　北京航空航天大学　\t   56.3   \n",
      "    12    \t　　　武汉大学　　　\t   56.2   \n",
      "    13    \t　　　同济大学　　　\t   55.7   \n",
      "    14    \t　　西安交通大学　　\t   55.0   \n",
      "    15    \t　　　四川大学　　　\t   54.4   \n",
      "    16    \t　　北京理工大学　　\t   54.0   \n",
      "    17    \t　　　东南大学　　　\t   53.6   \n",
      "    18    \t　　　南开大学　　　\t   52.8   \n",
      "    19    \t　　　天津大学　　　\t   52.3   \n",
      "    20    \t　　华南理工大学　　\t   52.0   \n",
      "    21    \t　　　中南大学　　　\t   50.3   \n",
      "    22    \t　　北京师范大学　　\t   49.7   \n",
      "    23    \t　　　山东大学　　　\t   49.1   \n",
      "    23    \t　　　厦门大学　　　\t   49.1   \n",
      "    25    \t　　　吉林大学　　　\t   48.9   \n",
      "    26    \t　　大连理工大学　　\t   48.6   \n",
      "    27    \t　　电子科技大学　　\t   48.4   \n",
      "    28    \t　　　湖南大学　　　\t   48.1   \n",
      "    29    \t　　　苏州大学　　　\t   47.3   \n",
      "    30    \t　　西北工业大学　　\t   46.7   \n"
     ]
    }
   ],
   "source": [
    "printUnivList(u_info, num=30) # 打印输出30个信息"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver  # 启动浏览器需要用到\n",
    "from selenium.webdriver.common.keys import Keys  # 提供键盘按键支持"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(\"http://www.python.org\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.quit()#关闭浏览器窗口"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "查找元素 定位到某一个具体的元素"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_name(\"q\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#进行交互\n",
    "element.send_keys(“some text”）  # 往一个可以输入对象中输入“some text”\n",
    "#甚至\n",
    "\n",
    "element.send_keys(Keys.RETURN）  # 模拟键盘回车\n",
    "#一般来说，这种方式输入后会一直存在，而要清空某个文本框中的文字，就需要：\n",
    "\n",
    "element.clear()  # 清空element对象中的文字"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在含有ajax加载的页面中 需要等待页面加载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "显示等待 触发某个条件后才能够执行后续的代码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()\n",
    "driver.get(\"http://somedomain/url_that_delays_loading\")\n",
    "try:    \n",
    "    element = WebDriverWait(driver, 10).until(           \n",
    "        EC.presence_of_element_located((By.ID, \"myDynamicElement\")))\n",
    "finally:    \n",
    "    driver.quit()\n",
    "#其中，presence_of_element_located是条件，By.ID是通过什么方式来确认元素（这个是通过id），\"myDynamicElement\"这个就是某个元素的ID"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "隐士等待 设置某个具体的等待时间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()\n",
    "driver.implicitly_wait(10) # seconds\n",
    "driver.get(\"http://somedomain/url_that_delays_loading\")\n",
    "myDynamicElement = driver.find_element_by_id(\"myDynami\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "session 与 cookie 模拟登陆163"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CDwindow-F754013AAB5C167414022C34B0337B19\n"
     ]
    }
   ],
   "source": [
    "name = \"yhz13064820396\"\n",
    "passwd = \"yhz15098437847\"\n",
    "driver = webdriver.Chrome()#固定格式\n",
    "driver.get(\"https://mail.163.com/\")\n",
    "driver.maximize_window()#最大化窗口\n",
    "time.sleep(5)\n",
    "current_window_1 = driver.current_window_handle#不知道干嘛的\n",
    "print(current_window_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"yhz\"\n",
    "passwd = \"yhz\"\n",
    "driver = webdriver.Chrome()#固定格式\n",
    "driver.get(\"https://mail.163.com/\")\n",
    "driver.maximize_window()#最大化窗口\n",
    "time.sleep(5)\n",
    "current_window_1 = driver.current_window_handle#不知道干嘛的\n",
    "print(current_window_1)\n",
    "\n",
    "#找到密码登录的模块 点击\n",
    "button = driver.find_element_by_id('lbNormal')\n",
    "button.click()\n",
    "\n",
    "#要先做整体的一个定位 driver.switch_to.frame 要操作哪个元素就要先进入此元素对应的frame中\n",
    "driver.switch_to.frame(driver.find_element_by_xpath(\"//iframe[starts-with(@id, 'x-URS-iframe')]\"))#做一个定位[@s属性，属性值]\n",
    "\n",
    "#输入账号和密码\n",
    "email = driver.find_element_by_name('email')\n",
    "#email = driver.find_element_by_xpath('//input[@name=\"email\"]')\n",
    "email.send_keys(name)\n",
    "password = driver.find_element_by_name('password')\n",
    "#password = driver.find_element_by_xpath(\"//input[@name='password']\")\n",
    "password.send_keys(passwd)\n",
    "submit = driver.find_element_by_id(\"dologin\")\n",
    "time.sleep(15)\n",
    "submit.click()\n",
    "time.sleep(10)\n",
    "print(driver.page_source)#打印网页的源代码\n",
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['HTTP://60.167.112.236:1133', 'HTTP://111.222.141.127:8118', 'HTTP://183.166.136.144:8888', 'HTTP://123.169.99.177:9999', 'HTTPS://114.99.54.65:8118', 'HTTP://58.253.158.177:9999', 'HTTP://112.84.73.53:9999', 'HTTPS://117.45.139.84:9006', 'HTTP://49.235.246.24:8118', 'HTTP://58.215.201.98:35728', 'HTTPS://121.13.252.60:41564', 'HTTP://112.95.205.138:8888', 'HTTP://218.27.204.240:8000', 'HTTPS://60.191.11.229:3128', 'HTTP://14.115.107.232:808', 'HTTPS://60.191.11.237:3128', 'HTTP://163.125.71.195:8888', 'HTTP://222.240.184.126:8086', 'HTTPS://218.22.7.62:53281', 'HTTPS://61.164.39.69:53281', 'HTTPS://121.13.252.60:41564', 'HTTPS://117.88.177.101:3000', 'HTTPS://114.99.54.65:8118', 'HTTPS://119.4.13.26:1133', 'HTTPS://117.45.139.84:9006', 'HTTPS://171.35.86.72:8118', 'HTTPS://114.223.103.47:8118', 'HTTPS://218.22.7.62:53281', 'HTTPS://61.164.39.69:53281', 'HTTPS://58.251.235.76:9797', 'HTTP://112.95.205.135:9000', 'HTTP://112.95.205.138:8888', 'HTTP://101.132.190.101:80', 'HTTP://183.166.136.144:8888', 'HTTP://123.169.99.177:9999', 'HTTP://163.125.71.198:9999', 'HTTP://58.253.158.177:9999', 'HTTP://112.84.73.53:9999', 'HTTP://49.235.246.24:8118', 'HTTP://123.163.24.113:3128']\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# 案例代码\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import time\n",
    "\n",
    "def open_proxy_url(url):\n",
    "    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'\n",
    "    headers = {'User-Agent': user_agent}\n",
    "    try:\n",
    "        r = requests.get(url, headers = headers, timeout = 20)\n",
    "        r.raise_for_status()\n",
    "        r.encoding = r.apparent_encoding\n",
    "        return(r.text)\n",
    "    except:\n",
    "        print('无法访问网页' + url)\n",
    "\n",
    "#使用bs4解析页面\n",
    "def get_proxy_ip(response):\n",
    "    proxy_ip_list = []\n",
    "    soup = BeautifulSoup(response, 'html.parser')\n",
    "    proxy_ips  = soup.select('.odd')#选择标签 选择所有同类的标签 selenium必要要先定位到frame模块\n",
    "    for proxy_ip in proxy_ips:\n",
    "        ip = proxy_ip.select('td')[1].text\n",
    "        port = proxy_ip.select('td')[2].text\n",
    "        protocol = proxy_ip.select('td')[5].text\n",
    "        if protocol in ('HTTP','HTTPS'):\n",
    "            proxy_ip_list.append(f'{protocol}://{ip}:{port}')\n",
    "    return proxy_ip_list\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    proxy_url = 'https://www.xicidaili.com/'\n",
    "    text = open_proxy_url(proxy_url)\n",
    "    proxy_ip_filename = 'proxy_ip.txt'\n",
    "    with open(proxy_ip_filename, 'w') as f:\n",
    "        f.write(text)\n",
    "    text = open(proxy_ip_filename, 'r').read()\n",
    "    proxy_ip_list = get_proxy_ip(text)\n",
    "    print(proxy_ip_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#寻找所有的tr标签\n",
    "def get_proxy_ip(response):\n",
    "    proxy_ip_list = []\n",
    "    soup = BeautifulSoup(response, 'html.parser')\n",
    "    proxy_ips = soup.find(id = 'ip_list').find_all('tr') \n",
    "    for proxy_ip in proxy_ips:\n",
    "        if len(proxy_ip.select('td')) >=8:\n",
    "            ip = proxy_ip.select('td')[1].text\n",
    "            port = proxy_ip.select('td')[2].text\n",
    "            protocol = proxy_ip.select('td')[5].text\n",
    "            if protocol in ('HTTP','HTTPS','http','https'):\n",
    "                proxy_ip_list.append(f'{protocol}://{ip}:{port}')\n",
    "    return proxy_ip_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用代理 直接传入到requests中的get方法就好\n",
    "    proxies的格式是一个字典：{‘http’: ‘http://IP:port‘,‘https’:'https://IP:port‘}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def open_url_using_proxy(url, proxy):\n",
    "    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'\n",
    "    headers = {'User-Agent': user_agent}\n",
    "    proxies = {}\n",
    "    if proxy.startswith('HTTPS'):\n",
    "        proxies['https'] = proxy\n",
    "    else:\n",
    "        proxies['http'] = proxy\n",
    "try:\n",
    "    r = requests.get(url, headers = headers, proxies = proxies, timeout = 10)\n",
    "    r.raise_for_status()\n",
    "    r.encoding = r.apparent_encoding\n",
    "    return (r.text, r.status_code)\n",
    "except:\n",
    "    print('无法访问网页' + url)\n",
    "    return False\n",
    "url = 'http://www.baidu.com'\n",
    "text = open_url_using_proxy(url, proxy_ip_list[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "确认ip代理地址的有效性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_proxy_avaliability(proxy):\n",
    "    url = 'http://www.baidu.com'\n",
    "    text, status_code = open_url_using_proxy(url, proxy)\n",
    "    VALID = False\n",
    "    if status_code == 200:\n",
    "        if r_title:\n",
    "            if r_title[0] == '<title>百度一下，你就知道</title>':\n",
    "                VALID = True\n",
    "    if VALID:\n",
    "        print('有效代理IP: ' + proxy)\n",
    "    else:\n",
    "        print('无效代理IP: ' + proxy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import re\n",
    "import json\n",
    "\n",
    "\n",
    "def open_proxy_url(url):\n",
    "    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'\n",
    "    headers = {'User-Agent': user_agent}\n",
    "    try:\n",
    "        r = requests.get(url, headers = headers, timeout = 10)\n",
    "        r.raise_for_status()\n",
    "        r.encoding = r.apparent_encoding\n",
    "        return r.text\n",
    "    except:\n",
    "        print('无法访问网页' + url)\n",
    "\n",
    "\n",
    "def get_proxy_ip(response):\n",
    "    proxy_ip_list = []\n",
    "    soup = BeautifulSoup(response, 'html.parser')\n",
    "    proxy_ips = soup.find(id = 'ip_list').find_all('tr')\n",
    "    for proxy_ip in proxy_ips:\n",
    "        if len(proxy_ip.select('td')) >=8:\n",
    "            ip = proxy_ip.select('td')[1].text\n",
    "            port = proxy_ip.select('td')[2].text\n",
    "            protocol = proxy_ip.select('td')[5].text\n",
    "            if protocol in ('HTTP','HTTPS','http','https'):\n",
    "                proxy_ip_list.append(f'{protocol}://{ip}:{port}')\n",
    "    return proxy_ip_list\n",
    "\n",
    "\n",
    "def open_url_using_proxy(url, proxy):\n",
    "    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'\n",
    "    headers = {'User-Agent': user_agent}\n",
    "    proxies = {}\n",
    "    if proxy.startswith(('HTTPS','https')):\n",
    "        proxies['https'] = proxy\n",
    "    else:\n",
    "        proxies['http'] = proxy\n",
    "\n",
    "    try:\n",
    "        r = requests.get(url, headers = headers, proxies = proxies, timeout = 10)\n",
    "        r.raise_for_status()\n",
    "        r.encoding = r.apparent_encoding\n",
    "        return (r.text, r.status_code)\n",
    "    except:\n",
    "        print('无法访问网页' + url)\n",
    "        print('无效代理IP: ' + proxy)\n",
    "        return False\n",
    "\n",
    "\n",
    "def check_proxy_avaliability(proxy):\n",
    "    url = 'http://www.baidu.com'\n",
    "    result = open_url_using_proxy(url, proxy)\n",
    "    VALID_PROXY = False\n",
    "    if result:\n",
    "        text, status_code = result\n",
    "        if status_code == 200:\n",
    "            r_title = re.findall('<title>.*</title>', text)\n",
    "            if r_title:\n",
    "                if r_title[0] == '<title>百度一下，你就知道</title>':\n",
    "                    VALID_PROXY = True\n",
    "        if VALID_PROXY:\n",
    "            check_ip_url = 'https://jsonip.com/'\n",
    "            try:\n",
    "                text, status_code = open_url_using_proxy(check_ip_url, proxy)\n",
    "            except:\n",
    "                return\n",
    "\n",
    "            print('有效代理IP: ' + proxy)\n",
    "            with open('valid_proxy_ip.txt','a') as f:\n",
    "                f.writelines(proxy)\n",
    "            try:\n",
    "                source_ip = json.loads(text).get('ip')\n",
    "                print(f'源IP地址为：{source_ip}')\n",
    "                print('='*40)\n",
    "            except:\n",
    "                print('返回的非json,无法解析')\n",
    "                print(text)\n",
    "    else:\n",
    "        print('无效代理IP: ' + proxy)\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    proxy_url = 'https://www.xicidaili.com/'\n",
    "    proxy_ip_filename = 'proxy_ip.txt'\n",
    "    text = open(proxy_ip_filename, 'r').read()\n",
    "    proxy_ip_list = get_proxy_ip(text)\n",
    "    for proxy in proxy_ip_list:#逐个检查ip的有效性\n",
    "        check_proxy_avaliability(proxy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "爬取丁香园留言板"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "因为目前对selenium库仍不是十分了解 所以到明天再回顾一下 再来完成练习 "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
